#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Fetch SoftRank (ShanghaiRanking) BCUR 2025 ranking JSON and download logos.
Outputs:
  - bcur_2025.json
  - bcur_2025.csv
  - logos/ (downloaded logo files)
"""
import os
import time
import csv
import json
import requests
import pandas as pd
from urllib.parse import urljoin

# 配置
BASE_DOMAIN = "https://www.shanghairanking.cn/"
API_URLS_TO_TRY = [
    # 常见 API 格式（网页前端和爬虫示例中使用的格式）
    "https://www.shanghairanking.cn/api/pub/v1/bcur?year=2025",
    "https://www.shanghairanking.cn/api/pub/v1/bcur?bcur_type=11&year=2025",
    # 备选：如果网站使用不同参数，可在此添加
]
OUT_JSON = "bcur_2025.json"
OUT_CSV = "bcur_2025.csv"
LOGO_DIR = "logos"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (compatible; DataCollector/1.0; +https://example.com)"
}

os.makedirs(LOGO_DIR, exist_ok=True)

def fetch_api(urls):
    for url in urls:
        try:
            print(f"尝试请求 {url}")
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                # 有些返回 application/json；有些可能返回 JSON 串在 text 中
                try:
                    data = r.json()
                except Exception:
                    data = json.loads(r.text)
                # 期待 data['data']['rankings'] 或 data['rankings']
                # 适配常见结构
                if isinstance(data, dict) and ("data" in data and "rankings" in data["data"]):
                    return data["data"]["rankings"], data
                if isinstance(data, dict) and "rankings" in data:
                    return data["rankings"], data
                # 有时返回嵌套层级不同，尝试常见键
                if isinstance(data, dict) and "data" in data:
                    for k in ("rankings", "list", "items"):
                        if k in data["data"]:
                            return data["data"][k], data
                # 否则如果是 list，也可能直接是排名列表
                if isinstance(data, list):
                    return data, data
                print("响应 JSON 中未找到预期字段，打印部分响应供排错：")
                print(json.dumps(data)[:2000])
                # 如果到这里，继续尝试下一个 URL
            else:
                print(f"HTTP {r.status_code} 来自 {url}")
        except Exception as e:
            print(f"请求 {url} 失败：{e}")
    raise RuntimeError("所有候选 API 都无法获取有效排名数据。")

def normalize_logo_url(logo_path: str) -> str:
    if not logo_path:
        return ""
    if logo_path.startswith("http"):
        return logo_path
    # 修复路径: 如果返回 "logo-jpg/" 或 "logo/", 换成 "_uni/logo-jpg/" 或 "_uni/logo/"
    if logo_path.startswith("logo-jpg/"):
        logo_path = "_uni/" + logo_path
    elif logo_path.startswith("logo/"):
        logo_path = "_uni/" + logo_path
    return urljoin(BASE_DOMAIN, logo_path.lstrip("/"))

# def download_logo(logo_path, out_filename):
#     # logo_path 可能是相对路径（例如 "logo/27532357.png"）或完整 URL
#     if not logo_path:
#         return False
#     if logo_path.startswith("http"):
#         full = logo_path
#     else:
#         full = urljoin(BASE_DOMAIN, logo_path.lstrip("/"))
#     try:
#         r = requests.get(full, headers=HEADERS, timeout=30, stream=True)
#         if r.status_code == 200:
#             with open(out_filename, "wb") as f:
#                 for chunk in r.iter_content(1024):
#                     f.write(chunk)
#             return True
#         else:
#             print(f"下载图标失败 HTTP {r.status_code} -> {full}")
#             return False
#     except Exception as e:
#         print(f"下载图标异常 {e} -> {full}")
#         return False

def download_logo(logo_path, out_filename):
    full = normalize_logo_url(logo_path)
    try:
        r = requests.get(full, headers=HEADERS, timeout=30, stream=True)
        if r.status_code == 200:
            with open(out_filename, "wb") as f:
                for chunk in r.iter_content(1024):
                    f.write(chunk)
            return True
        else:
            print(f"下载图标失败 HTTP {r.status_code} -> {full}")
            return False
    except Exception as e:
        print(f"下载图标异常 {e} -> {full}")
        return False

def main():
    rankings, raw = fetch_api(API_URLS_TO_TRY)
    print(f"共抓取到 {len(rankings)} 条记录（如果是分段/分页请检查 API）。")
    # 保存原始 JSON
    with open(OUT_JSON, "w", encoding="utf-8") as f:
        json.dump(raw, f, ensure_ascii=False, indent=2)
    # 规范化并保存 CSV
    rows = []
    for i, item in enumerate(rankings):
        # 常见字段映射（根据站点实际返回字段可调整）
        rank = item.get("rankOverall") or item.get("rank") or item.get("index") or (i+1)
        name_cn = item.get("univNameCn") or item.get("univName") or item.get("university_name") or ""
        name_en = item.get("univNameEn") or item.get("univNameEn") or ""
        slug = item.get("univUp") or item.get("univSlug") or ""
        logo = item.get("univLogo") or item.get("logo") or ""
        score = item.get("score") or item.get("totalScore") or ""
        category = item.get("univCategory") or item.get("category") or ""
        province = item.get("province") or ""
        tags = item.get("univTags") or item.get("tags") or []
        rows.append({
            "rank": rank,
            "name_cn": name_cn,
            "name_en": name_en,
            "slug": slug,
            "logo": logo,
            "score": score,
            "category": category,
            "province": province,
            "tags": "|".join(tags) if isinstance(tags, list) else tags
        })
    df = pd.DataFrame(rows)
    df.to_csv(OUT_CSV, index=False, encoding="utf-8-sig")
    print(f"已保存 CSV -> {OUT_CSV}")

    # 下载 logo
    for idx, row in df.iterrows():
        logo_path = row["logo"]
        if not logo_path:
            print(f"{row['rank']:>3} {row['name_cn']} 无 logo 字段，跳过。")
            continue
        ext = os.path.splitext(logo_path)[1] or ".png"
        safe_slug = row["slug"] or row["name_en"] or row["name_cn"]
        safe_slug = "".join(c if c.isalnum() or c in "-_" else "_" for c in safe_slug)[:80]
        outfn = os.path.join(LOGO_DIR, f"{str(row['rank']).zfill(2)}_{safe_slug}{ext}")
        if os.path.exists(outfn):
            print(f"已存在 {outfn}，跳过下载。")
            continue
        ok = download_logo(logo_path, outfn)
        if ok:
            print(f"下载 logo -> {outfn}")
        else:
            print(f"下载失败：{row['name_cn']} ({logo_path})")
        time.sleep(0.2)  # 轻微延迟，避免触发防护

if __name__ == "__main__":
    main()